<?php

class WebScrap{
	private $url;
	private $xpath;
  public $page;

	public function WebScrap($url,$xpath){
		$this->url = $url;
		$this->xpath = $xpath;
    $this->page = "";
	}
  
  public function SetXpath($xpath){
    $this->xpath = $xpath;
  }

	public function GetScrap(){
    $aContent = array();
		// use Tidy to try to make the page well formed
    if (empty($this->page)){
      $this->page = $this->page = $this->TidyIt($this->url);
      $this->page = mb_convert_encoding($this->page, 'HTML-ENTITIES', "UTF-8"); 
    }
		// create a document out of the well formed content
		$domDocument=new DOMDocument('1.0');
		@$domDocument->loadHTML($this->page);

		// create an XPath object out of the document and query it for the supplied xpath
		$domXPath = new DOMXPath($domDocument);
    
    if (is_array($this->xpath)){
      foreach ($this->xpath as $title=>$xpath){
        $domNodeList = $domXPath->query($xpath);
    		// Get the content (HTML) out of the NodeList returned by the DOMXPath::query
    		$content = $this->GetHTMLFromNodeList($domNodeList);
        foreach ($content as $key=>$c){
          $aContent[$key][$title] = $c;
        }
      }

      return $aContent;
    }else{
      $domNodeList = $domXPath->query($this->xpath);
    	return $this->GetHTMLFromNodeList($domNodeList);
    }
  }
  
  private function GetPageContent($url){
    $ch = curl_init($url);
    curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_ENCODING, "UTF-8");
    curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 1000);
    curl_setopt($ch, CURLOPT_REFERER, $url);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; rv:12.0) Gecko/20100101 Firefox/12.0');
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
    $output = curl_exec($ch); 
    $info = curl_getinfo($ch);
    curl_close($ch);
    
    if ($output === false || $info['http_code'] != 200){
      sleep(30);
      return false;
    }
    return $output;
  }

	private function TidyIt($url){
    $content = false;
    while ($content===false) $content = $this->GetPageContent($url);
    
    return $content;
		/*$tidy = new tidy();
		$tidy->parseString($this->GetPageContent($url), array(), "utf8");
		$tidy->cleanRepair();
		return $tidy;*/
	}

	private function GetHTMLFromNodeList($domNodeList){    
    $aNode = array();
    for ($i=0; $i<$domNodeList->length; $i++){
      $node = $domNodeList->item($i);
      $domDocument = new DOMDocument();
      foreach($node->childNodes as $childNode)
        $domDocument->appendChild($domDocument->importNode($childNode, true));
        
      $aNode[] = trim($domDocument->saveHTML());
    }
    
    return $aNode;
  }
}

?>